<?php
/**
 * Created by PhpStorm.
 * User: xpz
 * Date: 2023/7/5
 * Time: 8:02
 */

namespace app\api\command\spider;
use app\api\command\spider\SpiderBase;
use think\console\Command;
use think\console\Input;
use think\console\input\Option;
use think\console\Output;
use think\Exception;
use think\config;
use think\Db;
use QL\QueryList;
class Mw extends SpiderBase
{
    private $baseDomain;
    protected function configure()
    {
        $this->setName('mw')              		//定义命令的名字
        ->setDescription('mw')      //定义命令的描述
        ->addArgument('method');                   //参数1
    }

    //    php think mw spider
    protected function execute(Input $input, Output $output)
    {
        $this->baseDomain = "https://www.duanmeiwen.com";
        require EXTEND_PATH.'phpQuery/phpQuery.php';
        ini_set('max_execution_time', 600);//秒为单位，自己根据需要定义
        $method = $input->getArgument('method');
        if($method == 'test'){
            ini_set("memory_limit","500M");
            $url = 'https://www.duanmeiwen.com/fanwen/shuxin/3535474.html';
//            $url = 'https://www.duanmeiwen.com/yulu/1826312.html';
            $this->test($url);
        } else if($method == 'index'){
            $url = 'https://www.duanmeiwen.com/xinshang/';
            $this->index($url);
        } else if($method == 'test1'){
            $this->test1();
        } else if($method == 'sync'){
            $this->sync();
        } else if($method == 'daoshuj'){
            $this->daoshuj();
        } else if($method == 'getcategory'){
           $this->getcategory();
        } else if($method == 'buquancategory'){
            $this->buquancategory();
        } else if($method == 'test_baidu'){
            $this->test_baidu();
        }

    }

    private function test_baidu(){
        $url = "https://www.baidu.com/s?wd=%E5%9B%9B%E5%BC%A6%E5%90%89%E4%BB%96%E5%85%A5%E9%97%A8%E6%8C%87%E6%B3%95%E5%9B%BE";
        $html = _get_url($url);
        $rules = [
            'name'=>['tr > td > a > span','text'],
        ];
        $range = '.rs-table_3RiQc tbody ';
        $data =  QueryList::html($html)->rules($rules)->range($range)->query()->getData()->all();
        print_r($data);
    }



    private function index($url){

//        $rules = [
//            'name' => ['a','href'],
//        ];
////        $range = '.main';
//        $data = QueryList::get($url)->rules($rules)->query()->getData();
//        $data = QueryList::get($url)->rules($rules)->range($range)->query()->getData()->all();
        $html = _get_url($url);
        $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
        $content = QueryList::html($html)->find('.content')->html();
//        $data = $ql->find('.two img')->attrs('alt');
//        print_r($data);
    }

    //131994 开始

    /**
     * 最新版本
     * @param $url
     */
    private function test($url){
        echo $url."抓96919取开始+++++++++++++++++++++++\r\n";
        if($this->_checkSpider($url)){
            $html = _get_url($url);

            $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
//            var_dump($html);exit;
            $content = QueryList::html($html)->find('.content')->html();
            $urlArr = parse_url($url);
            $pathArr = explode('/',$urlArr['path']);

            array_shift($pathArr);
            array_pop($pathArr);

            $pathString = '';
            if(!empty($pathArr)){
                $pathString = implode(',',$pathArr);
            }

            $bigTitle = QueryList::html($html)->find('h1')->html();
//            var_dump($bigTitle);exit;

            if(strpos($content,'</h2>') !== false){
                $content = str_replace([' style="white-space: normal;"'],[''],$content);
//                echo $content;
                $content = str_replace(['<h2>','</h2>'],['<p>#','#</p>'],$content);
//                echo $content;exit;
                $contentArr = QueryList::html($content)->find('p')->htmls();
//                print_r($contentArr);exit;
                $return = [];
                $image = '';
                $xiangguanArr = [];
                if(!empty($contentArr)){
                    foreach($contentArr as $k=>$v){
//                        var_dump($v);
//                        $v = $this->myTrim($v);
//                        echo $v."\r\n";
                        //处理图片
                        if(strpos($v,'<img') !== false){
                            $imageUrl = QueryList::html($v)->find('img')->attr('src');
                            $imageUrl = str_replace('https','http',$imageUrl);
                            $image = $this->_mwDownImage($imageUrl,'article/'.date('Ymd'));
                            continue;
                        }
                        //处理相关链接
                        if(strpos($v,'<a') !== false){
//                            echo '3';
                            $xiangguanArr[] = QueryList::html($v)->find('a')->attr('href');
                            continue;
                        }


                        //处理标题做key
//                        echo $v."\r\n";
                        $firest = mb_substr($v,0,1);
//                        var_dump($firest);
                        $end = mb_substr($v,-1,1);
//                        echo $firest;exit;
                        if($firest == '#' && $end == '#'){
//                            echo "2222";
                            $key = $v;
                        }
                        if(!isset($key)){
//                            echo "1";
                            continue;
                        }
//                        var_dump($key);exit;
//                        var_($v);
                        $return[$key][] = $v;
                    }
//                    print_r($return);exit;
                    if(!empty($return)){
                        foreach($return as $k=>$v){
                            $title = array_shift($v);
                            $content = '';
                            if(!empty($v)){
                                foreach($v as $k=>$v){
                                    if(strpos($v,'相关文章') == false){
                                        $content .="<p>".$this->myTrim($v)."</p>";
                                    }

                                }
                            }
                            if(empty($content)){
                                continue;
                            }
                            $data = [
                                'title'=>myTrim(strip_tags(str_replace('#',"",$title))),
                                'content' => $content,
                                'source_url' => $url,
                                'url_md5' => md5($url),
                                'image' => $image,
                                'type' => $pathString,
                                'weigh' => rand(0,99999999),
                                'big_title' => $bigTitle,
                                'is_single' => 2,
                            ];
//                            print_r($data);exit;
                            $this->insert($data);
                        }

                    }
                    echo $url."抓取结束==============================\r\n";
//                    print_r($xiangguanArr);exit;
                    //抓取相关文章
                    if(!empty($xiangguanArr)){
                        foreach($xiangguanArr as $k=>$v){
//                            echo $v."\r\n";
                            if(!empty($v)){

                                if(mb_substr($v,-4) == 'html'){
                                    $this->test($v);
                                }

                            }

                        }
                    }
                }
            } else {
//                echo $url."暂时不抓取\r\n";
                $this->single($url);
            }
        } else {
            $articleModel = Model('app\common\model\ArticleSpider');

            $articleData = $articleModel->where(['url_md5'=>md5($url)])->find();
//            var_dump($articleData);exit;
//            var_dump($articleData);exit;
            if(empty($articleData['big_title'])){
                $html = _get_url($url);
                $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
                $bigTitle = QueryList::html($html)->find('h1')->html();
                $content = QueryList::html($html)->find('.content')->html();
                if(strpos($content,'</h2>') !== false){
                    $contentArr = QueryList::html($content)->find('p')->htmls();
                    $xiangguanArr = [];
                    if(!empty($contentArr)) {
                        foreach ($contentArr as $k => $v) {
                            //处理相关链接
                            if(strpos($v,'<a') !== false){
                                $xiangguanArr[] = QueryList::html($v)->find('a')->attr('href');
                                continue;
                            }
                        }
                    }
                }
                if(!empty($bigTitle)){
                    $articleModel->where(['url_md5'=>md5($url)])->update(['big_title'=>$bigTitle]);
                    echo "\r\n";
                    echo $articleData['id']."更新big_title标题成功【".$bigTitle."】\r\n";
                    echo "\r\n";
                }

                //抓取相关文章
                if(!empty($xiangguanArr)){
                    foreach($xiangguanArr as $k=>$v){
//                            echo $v."\r\n";
                        $this->test($v);
                    }
                }
            }
            echo $url."test已经抓取过\r\n";
        }
    }


    private function single($url){
        echo $url."抓取single开始+++++++++++++++++++++++\r\n";
        if($this->_checkSpider($url)){
            $html = _get_url($url);
            $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
            $urlArr = parse_url($url);
            $pathArr = explode('/',$urlArr['path']);
            array_shift($pathArr);
            array_pop($pathArr);
            $pathString = '';
            if(!empty($pathArr)){
                $pathString = implode(',',$pathArr);
            }
            $bigTitle = QueryList::html($html)->find('h1')->html();
            $content = QueryList::html($html)->find('.content')->html();
            $contentArr = QueryList::html($content)->find('p')->htmls();
//        print_r($contentArr);exit;
            if(!empty($contentArr)){
                $contentString = '';
                $image = '';
                foreach($contentArr as $k=>$v){
                    if($v == $bigTitle){
                        continue;
                    }
                    //处理图片
                    if(strpos($v,'<img') !== false){
                        $imageUrl = QueryList::html($v)->find('img')->attr('src');
                        $imageUrl = str_replace('https','http',$imageUrl);
                        $image = $this->_mwDownImage($imageUrl,'article/'.date('Ymd'));
                        continue;
                    }
                    //处理相关链接
                    if(strpos($v,'<a') !== false){
//                    echo "asdf";exit;
                        $xiangguanArr[] = QueryList::html($v)->find('a')->attr('href');
                        continue;
                    }

                    if(strpos($v,'相关文章') == false){
                        $contentString .="<p>".$this->myTrim($v)."</p>";
                    }

                }
//                print_r($xiangguanArr);exit;

                $data = [
                    'title'=>strip_tags(str_replace('#',"",$bigTitle)),
                    'content' => $contentString,
                    'source_url' => $url,
                    'url_md5' => md5($url),
                    'image' => $image,
                    'type' => $pathString,
                    'weigh' => rand(0,99999999),
                    'big_title' => $bigTitle,
                    'is_single' => 1,
                ];
//                print_r($data);exit;
                $this->insert($data);


                //抓取相关文章
                if(!empty($xiangguanArr)){
                    foreach($xiangguanArr as $k=>$v){
//                            echo $v."\r\n";
                        if(!empty($v)){

                            if(mb_substr($v,-4) == 'html'){
                                $this->test($v);
                            }

                        }

                    }
                }
            }
        } else {
            echo $url."single已经抓取过\r\n";
        }


    }



    private function xiangguan($url){
        $rules = [
            'name' => ['.char-info>a','text'],
            'real_name'=>['.char-info>div>.black_link','text'],
            'pingfen'=>['.yj-score','text'],
            'image'=>['.char-img','src'],
            'jieshao'=>['.char-content','text'],
        ];
        $range = '.ac_piclst li';
//        $data = QueryList::get($url)->rules($rules)->query()->getData();
        $data = QueryList::get($url)->rules($rules)->range($range)->query()->getData()->all();
    }

    private function insert($data){
        $articleModel = Model('app\common\model\ArticleSpider');
        $content = $data['content'];
        unset($data['content']);
        $id = $articleModel->insertGetId($data);
        $contentArr = [
          'aid'=>$id,
          'content'=>$content,
        ];
        Model('app\common\model\ArticleSpiderContent')->insertGetId($contentArr);

    }
    private function _checkSpider($url){
        $articleModel = Model('app\common\model\ArticleSpider');
        // var_dump($articleModel);exit;
        $articleData = $articleModel->where(['url_md5'=>md5($url)])->find();
        if(empty($articleData)){
            return true;
        } else {
            return false;
        }
        // var_dump($articleData);
    }

    private function _mwDownImage($image,$dir){
        $defaultArr = [
            'https://static2.tvzhe.com/images/style/obj_def.jpg'
        ];
        if(in_array($image,$defaultArr)){
            return "";
        }
        return $this->_downImage($image,$dir);
    }

    private function test1(){

        $sql = "SELECT id,title  FROM `xpz_article_spider` where  char_length(title)>30";
        $rs = Db::query($sql);
        if(!empty($rs)){
            $i = 0;
            foreach($rs as $k=>$v){
//                if(strpos($v['title'],'<img') !== false){
//                    echo $v['id']."\r\n";
                $i++;
                $sql = "DELETE FROM `xpz_article_spider` WHERE id=".$v['id'];
                Db::execute($sql);
//                }
            }
            echo $i;
        }


        exit;



        $sql = "select * from xpz_article_spider order by id desc limit 10";
        $rs = Db::query($sql);
        if(!empty($rs)){
            $articleModel = Model('app\common\model\Article');
            foreach($rs as $k=>$v){
                $data = [
                    'title'=>$v['title'],
                    'content' => $v['content'],
                    'image' => $v['image'],
                    'summary' => str_cut($v['content']),
                ];
//                            print_r($data);
                $rs = $articleModel->insertGetId($data);
                if(!empty($rs)){
                    $sql = "UPDATE `xpz_article_spider` SET `is_use`=2 where id =".$v['id'];
                    Db::execute($sql);
                }
            }
        }
    }

    private function sync(){
        $sql = "select * from xpz_article_spider where is_use = 1 order by weigh desc limit 5";
        $rs = Db::query($sql);
        if(!empty($rs)){
            $articleModel = Model('app\common\model\Article');
            foreach($rs as $k=>$v){
                $sqlContent = "select * from xpz_article_spider_content where aid = ".$v['id'];
                $rsContent = Db::query($sqlContent);
                $data = [
                    'title'=>$v['title'],
                    'content' => $rsContent[0]['content'],
                    'image' => $v['image'],
                    'summary' => str_cut($rsContent[0]['content']),
                    'add_time' => time(),
                ];
//                            print_r($data);exit;
                $rs = $articleModel->insertGetId($data);
                if(!empty($rs)){
                    $sql = "UPDATE `xpz_article_spider` SET `is_use`=2 where id =".$v['id'];
                    Db::execute($sql);
                    echo $v['id']."同步成\r\n";
                }
            }
        }
    }


    public function daoshuj(){
        $sql = "select * from xpz_article_spider where is_use = 1 order by id asc limit 250000,30000";
        $rs = Db::query($sql);
        if(!empty($rs)){
            $articleModel = Model('app\common\model\ArticleSpiderContent');
            foreach($rs as $k=>$v){
                $articleData = $articleModel->where(['aid'=>$v['id']])->find();
                if(empty($articleData)){
                    $data = [
                        'aid'=>$v['id'],
                        'content' => $v['content'],
                    ];
                    $id = $articleModel->insertGetId($data);
                    echo $id."成功导入\r\n";
                } else {
                    echo $articleData['aid']."已存在\r\n";
                }

            }
        }
    }

    public function getcategory(){
        $sql = "SELECT type,count(id) as c FROM `xpz_article_spider` group by type order by c desc";
        $rs = Db::query($sql);
        if(!empty($rs)){
            $categoryModel = Model('app\common\model\Category');
            foreach($rs as $k=>$v){
                if(strstr($v['type'],',') == false){//一级
                    $categoryData = $categoryModel->where(['pid'=>0,'nickname'=>$v['type']])->find();
                    if(empty($categoryData)){
//                        echo "一级分类开始：\r\n";
                        echo $v['type']."==".$v['c']."\r\n";
//                        echo "一级分类结束\r\n";
                    }
                } else {
//                    if($v['c'] < 11){
//                        continue;
//                    }
                    echo $v['type']."\r\n";
                    $typeArr = explode(',',$v['type']);
                    $categoryData = $categoryModel->where(['pid'=>0,'nickname'=>$typeArr[0]])->find();
                    if(empty($categoryData)){
                        echo "一级分类：【".$typeArr[0]."】不存在！=>";
                    } else {
                        $categoryData2 = $categoryModel->where(['pid'=>$categoryData['id'],'nickname'=>$typeArr[1]])->find();
                        if(empty($categoryData2)){
                            echo "一级分类：【".$categoryData['nickname']."】存在！二级分类【".$typeArr[1]."】不存在=>".$v['c']."\r\n";
                        }
                    }


                }

            }
        }
    }

    public function buquancategory(){
//        $sql = "SELECT * FROM `xpz_category` where pid = 0";
//        $rs = Db::query($sql);
//        if(!empty($rs)){
//            foreach($rs as $k=>$v){
//                $sql2 = "select * from xpz_article_spider where type = '".$v['nickname']."' limit 1";
//                $rs2 = Db::query($sql2);
//                if(isset($rs2[0]['source_url'])){
//                    $html = _get_url($rs2[0]['source_url']);
//                    $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
////                    echo $html;exit;
//                    $h1text = QueryList::html($html)->find('.source')->text();
//                    echo $v['nickname']."\r\n";
//                    echo $rs2[0]['source_url']."\r\n";
//                    echo $h1text."\r\n";
//                    $updateSql = "UPDATE `xpz_category` SET `diyname`= '".$h1text."' where id = ".$v['id'];
//                    echo $updateSql."\r\n";
//                    Db::execute($updateSql);
//                }
//
//            }
//        }


        $sql = "SELECT * FROM `xpz_category` where pid = 0";
        $rs = Db::query($sql);
        if(!empty($rs)){
            foreach($rs as $k=>$v){
                $sql3 = "select * from xpz_category where pid=".$v['id'];
                $rs3 = Db::query($sql3);
                if(!empty($rs3)){
                    foreach($rs3 as $key=>$val){
                        $sql2 = "select * from xpz_article_spider where type = '".$v['nickname'].",".$val['nickname']."' limit 1";
                        $rs2 = Db::query($sql2);
                        if(isset($rs2[0]['source_url'])){
                            $html = _get_url($rs2[0]['source_url']);
                            $html=mb_convert_encoding($html, 'UTF-8', 'UTF-8,GBK,GB2312,BIG5');
//                    echo $html;exit;
                            $h1text = QueryList::html($html)->find('.source')->text();
                            echo $v['nickname']."\r\n";
                            echo $rs2[0]['source_url']."\r\n";
                            echo $h1text."\r\n";
                            $updateSql = "UPDATE `xpz_category` SET `diyname`= '".$h1text."' where id = ".$val['id'];
                            echo $updateSql."\r\n";
                            Db::execute($updateSql);
                        }
                    }
                }

            }
        }




    }


}

